﻿/* 
* Copyright (c) 2009, The University of Texas at Austin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, 
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, 
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice, 
* this list of conditions and the following disclaimer in the documentation and/or other materials 
* provided with the distribution.
*
* Neither the name of The University of Texas at Austin nor the names of its contributors may be 
* used to endorse or promote products derived from this software without specific prior written 
* permission.
* 
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR 
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 
* THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/


using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Bio.Data.Interfaces;
using System.IO.MemoryMappedFiles;

// AE2 Header documentation
// nobase, locked, none, tobase -- these are the editing modes that ae2 supports.  2x2 matrix of editable/not and nucleotides/annotation.
// nobase -- annotation editable, nucleotides not (default)
// locked -- annotation and nucleotides not editable
// none -- annotation and nucleotides editable
// tobase -- nucleotides editable, annotation not (really, no idea when this is useful)
// in or -in -- "include."  Used by external programs (most notably, query) to determine which sequences to consider for analysis.  Unaligned sequences should always be -in.
// out or -out -- "output."  Determines if a sequence is affected by AE2's internal algorithms.  For example, in the ae2 "diff" mode, where identical nucleotides are replaced by dots, it's often useful to have one sequence NOT appear as dots, so you know what the "baseline" for the diff is...
// vis or -vis -- "visibility."  AE2's closest analogue to CATUI views.
// prt or bold -- actually, prt, -prt, or bold.  Printing within AE2.  "Bold" is a special case of "print" here.  We rarely use excerpts from alignments as figures anymore, which I suspect was the original impetus for the feature.
// dna or rna -- Changes "T" (DNA) to "U" (RNA) and vice versa.  If a sequence is out of sync (i.e., labeled as dna but contains "U"), swapping back and forth will restore the synchrony.
// lin or cir -- linear or circular sequences.  In practice, I've never used this, but in theory it is useful to draw this distinction.
// 5> or 3< -- direction of the sequence.  In AE2, this toggle functions on its own, reversing only the order of the sequence (i.e., ACGUGGGC becomes CGGGUGCA).
// func or comp -- functional or complementary strand.  Which of the two DNA strands contains the gene of interest.  In AE2, this toggle only functions in conjunction with the 5>/3< toggle (i.e., toggling the func of "5> func" changes to "3< comp" , likewise with "3< func" to "5> comp").  This leads to another 2x2 matrix of possibilities:
// 5' func -- the gene of interest lies in the strand described in the forward orientation.
// 3' func -- never used? 
// 5' comp -- aka "reverse complement" -- the gene of interest lies in the forward orientation of the reverse complement of the strand described.
// 3' comp -- never used?
// ref  = reference sequence.  One per alignment in AE2.

namespace Bio.Data.Providers.Alignment.AE2
{
    /// <summary>
    /// This class is used to load a set of RNA sequences from an AE2 data file.
    /// It loads the entire sequence into memory - assuming that the file is not too large to manage due
    /// to AE2 restrictions.
    /// </summary>
    public class AE2File : BioFileDataLoader<IBioSequence>
    {
        private const string PrivateKey = @"AE2!";
        private const string SeqdataRowRegex = @"(?<Rowname>[^\n\s]+)\s+(?<Startindex>\d+)\s+(?<Seqdata>[^\n\s]+)";

        private const int SmallFileSize = 10 * 1024 * 1024; // 10M
        internal MemoryMappedFile MmFile { get; private set; }

        /// <summary>
        /// This method is used to prepare the loader to access the
        /// Entities collection.
        /// </summary>
        protected override void OnLoad()
        {
            // Debug: special flag to load entire file into memory.  Place into 
            // <appSettings /> in the app.config file of controlling application.
            string value = ConfigurationManager.AppSettings["AE2File:LoadIntoMemory"];
            bool loadIntoMemory = false;
            if (!string.IsNullOrEmpty(value))
                Boolean.TryParse(value, out loadIntoMemory);
            if (!loadIntoMemory)
                loadIntoMemory = (new FileInfo(InitializationData).Length < SmallFileSize);

            if (loadIntoMemory)
            {
                LoadEntireFileIntoMemory();
            }
            else
            {
                IndexForVirtualization();
            }
        }

        /// <summary>
        /// This method loads the entire AE2 file into memory
        /// </summary>
        private void LoadEntireFileIntoMemory()
        {
            Dictionary<string, SequenceMetadata> metaData = new Dictionary<string, SequenceMetadata>();
            var bioValidator = BioValidator.rRNAValidator;

            using (var tr = new StreamReader(InitializationData))
            {
                string line = tr.ReadLine();
                while (!string.IsNullOrEmpty(line))
                {
                    // Header metadata?
                    if (line.StartsWith("#="))
                    {
                        string[] headerData = line.Split(' ').Where(l => !string.IsNullOrEmpty(l)).ToArray();
                        string rowName = headerData[1].Trim();
                        SequenceMetadata mdata = new SequenceMetadata()
                         {
                             IsVisible = string.Compare(headerData[5].Trim(), "vis", true) == 0,
                             IsReferenceSequence = headerData.Length > 11 && string.Compare(headerData[11].Trim(), "ref", true) == 0
                         };

                        // Dump all the stuff we don't care about into metadata.
                        for (int i = 0; i < 9; i++)
                            mdata.Data[PrivateKey + i] = new SequenceMetadataElement(headerData[i + 2].Trim(), true);

                        metaData.Add(rowName, mdata);
                    }
                    // Comment header?
                    else if (line.StartsWith("#:"))
                    {
                        string[] data = line.Split(':');
                        if (data.Length >= 3)
                        {
                            string rowName = data[1];
                            SequenceMetadata metaMatch;
                            if (metaData.TryGetValue(rowName, out metaMatch))
                            {
                                string key = data[2];
                                string commentLine = string.Join(":", data.Skip(3).Select(s => s).ToArray()); ;
                                if (metaMatch.Data.ContainsKey(key))
                                    metaMatch.Data[key] = metaMatch.Data[key] + Environment.NewLine + commentLine;
                                else
                                    metaMatch.Data[key] = commentLine;
                            }
                        }

                    }
                    // Sequence data
                    else if (!line.StartsWith("#"))
                    {
                        Match regexMatch = Regex.Match(line, SeqdataRowRegex);
                        if (regexMatch.Success)
                        {
                            // Get the row name
                            string rowName = regexMatch.Groups["Rowname"].Value;

                            // Get the starting index of the data
                            int startIndex = Int32.Parse(regexMatch.Groups["Startindex"].Value);

                            // .. and the sequence data itself.
                            string seqData = regexMatch.Groups["Seqdata"].Value;

                            // We assume the data itself is loaded in order - i.e. sequence data is consecutive.
                            SimpleRNASequence sequence = (Entities.Count > 0) ? (SimpleRNASequence)Entities[Entities.Count - 1] : null;
                            if (sequence == null || string.Compare(sequence.Name, rowName) != 0)
                            {
                                sequence = new SimpleRNASequence(bioValidator) { Name = rowName, ScientificName = rowName };
                                Entities.Add(sequence);

                                SequenceMetadata metaMatch;
                                metaData.TryGetValue(rowName, out metaMatch);
                                if (metaMatch != null)
                                {
                                    IExtensibleProperties props = sequence as IExtensibleProperties;
                                    if (props != null)
                                        props.ExtendedProperties[SequenceMetadata.Key] = metaMatch;
                                }
                            }

                            // AE2 omits the blanks so skip anything we haven't seen with a gap.
                            if (sequence.SequenceData.Count < startIndex)
                            {
                                for (int i = 0; i < (startIndex - sequence.SequenceData.Count); i++)
                                    sequence.AddSymbol(BioSymbol.Gap);
                            }

                            for (int index = 0; index < seqData.Length; )
                            {
                                IBioSymbol symbol;
                                if (seqData[index] == '\\')
                                {
                                    symbol = CharacterToBioSymbol(bioValidator, ConvertOctal(new[] { seqData[index + 1], seqData[index + 2], seqData[index + 3] }));
                                    index += 4;
                                }
                                else
                                {
                                    symbol = CharacterToBioSymbol(bioValidator, seqData[index]);
                                    index++;
                                }
                                if (symbol != null)
                                    sequence.AddSymbol(symbol);
                            }
                        }
                    }
                    line = tr.ReadLine();
                }
            }

            // Force the data length to be consistent for all sequences.
            int maxCount = Entities.Max(e => e.SequenceData.Count);
            Parallel.ForEach(Entities, (e) =>
               {
                   if (e.SequenceData.Count < maxCount)
                   {
                       for (int i = 0; i < maxCount - e.SequenceData.Count; i++)
                           ((SimpleRNASequence)e).AddSymbol(BioSymbol.None);
                   }
               });
        }

        /// <summary>
        /// This method indexes the AE2 file for virtualization support
        /// </summary>
        private void IndexForVirtualization()
        {
            Dictionary<string, SequenceMetadata> metaData = new Dictionary<string, SequenceMetadata>();
            var bioValidator = BioValidator.rRNAValidator;

            MmFile = MemoryMappedFile.CreateFromFile(InitializationData, FileMode.Open);
            using (var fs = MmFile.CreateViewStream())
            {
                string line = fs.ReadLine();
                while (!string.IsNullOrEmpty(line))
                {
                    // Header metadata?
                    if (line.StartsWith("#="))
                    {
                        string[] headerData = line.Split(' ').Where(l => !string.IsNullOrEmpty(l)).ToArray();
                        string rowName = headerData[1].Trim();
                        SequenceMetadata mdata = new SequenceMetadata()
                        {
                            IsVisible = string.Compare(headerData[5].Trim(), "vis", true) == 0,
                            IsReferenceSequence = headerData.Length > 11 && string.Compare(headerData[11].Trim(), "ref", true) == 0
                        };

                        // Dump all the stuff we don't care about into metadata.
                        for (int i = 0; i < 9; i++)
                            mdata.Data[PrivateKey + i] = new SequenceMetadataElement(headerData[i + 2].Trim(), true);

                        metaData.Add(rowName, mdata);
                    }
                    // Comment header?
                    else if (line.StartsWith("#:"))
                    {
                        string[] data = line.Split(':');
                        if (data.Length >= 3)
                        {
                            string rowName = data[1];
                            SequenceMetadata metaMatch;
                            if (metaData.TryGetValue(rowName, out metaMatch))
                            {
                                string key = data[2];
                                string commentLine = string.Join(":", data.Skip(3).Select(s => s).ToArray()); ;
                                if (metaMatch.Data.ContainsKey(key))
                                    metaMatch.Data[key] = metaMatch.Data[key] + Environment.NewLine + commentLine;
                                else
                                    metaMatch.Data[key] = commentLine;
                            }
                        }

                    }
                    // Sequence data
                    else if (!line.StartsWith("#"))
                    {
                        Match regexMatch = Regex.Match(line, SeqdataRowRegex);
                        if (regexMatch.Success)
                        {
                            // Get the row name
                            string rowName = regexMatch.Groups["Rowname"].Value;

                            // Get the starting index of the data
                            int startIndex = Int32.Parse(regexMatch.Groups["Startindex"].Value);

                            // .. and the sequence data itself.
                            string seqData = regexMatch.Groups["Seqdata"].Value;

                            // We assume the data itself is loaded in order - i.e. sequence data is consecutive.
                            VirtualizedAE2Sequence sequence = (Entities.Count > 0) ? (VirtualizedAE2Sequence)Entities[Entities.Count - 1] : null;
                            if (sequence == null || string.Compare(sequence.Name, rowName) != 0)
                            {
                                // Mark the ending position
                                if (sequence != null)
                                {
                                    long endPos = fs.Position - (1 + line.Length);
                                    sequence._blockLength = endPos - sequence._dataPositions[0].Item2;
                                }

                                // Create the new sequence
                                sequence = new VirtualizedAE2Sequence(this, bioValidator) { Name = rowName, ScientificName = rowName };
                                Entities.Add(sequence);

                                // Add the metadata
                                SequenceMetadata metaMatch;
                                metaData.TryGetValue(rowName, out metaMatch);
                                if (metaMatch != null)
                                {
                                    IExtensibleProperties props = sequence as IExtensibleProperties;
                                    if (props != null)
                                        props.ExtendedProperties[SequenceMetadata.Key] = metaMatch;
                                }
                            }

                            // Add known position
                            sequence._dataPositions.Add(new Tuple<int, long>(startIndex, fs.Position - (1 + seqData.Length)));

                            // Count the nucleotides here.
                            int count = 0;
                            for (int index = 0; index < seqData.Length; count++)
                            {
                                if (seqData[index] == '\\')
                                    index += 4;
                                else
                                    index++;
                            }
                            
                            sequence._dataCount += count;

                        }
                    }
                    line = fs.ReadLine();
                }
            }

            // Force the data length to be consistent for all sequences.
            int maxCount = Entities.Max(e => ((VirtualizedAE2Sequence)e)._dataCount);
            Parallel.ForEach(Entities, (e) => ((VirtualizedAE2Sequence) e).ForceCount(maxCount));
        }

        /// <summary>
        /// This method converts an AE2 character into a BioSymbol.
        /// </summary>
        /// <param name="bioValidator"></param>
        /// <param name="ch"></param>
        /// <returns></returns>
        internal static IBioSymbol CharacterToBioSymbol(IBioValidator bioValidator, char ch)
        {
            return (ch == '~')
                ? BioSymbol.None
                : (ch == '-') ? BioSymbol.Gap
                : bioValidator.IsValid(ch)
                             ? new BioSymbol(BioSymbolType.Nucleotide, ch)
                             : new BioSymbol(BioSymbolType.Unknown, ch);
        }

        /// <summary>
        /// Converts an Octal (base8) number to base10 character.
        /// </summary>
        /// <param name="octal"></param>
        /// <returns></returns>
        internal static char ConvertOctal(char[] octal)
        {
            int one = (Int32.Parse(Char.ToString(octal[0])));
            int two = (Int32.Parse(Char.ToString(octal[1])));
            int three = (Int32.Parse(Char.ToString(octal[2])));
            int val = 64 * one + 8 * two + three;
            return Convert.ToChar(val - 128);
        }
    }
}